{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "67559d96",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T08:20:52.248396Z",
     "iopub.status.busy": "2024-10-06T08:20:52.247662Z",
     "iopub.status.idle": "2024-10-06T08:21:06.205977Z",
     "shell.execute_reply": "2024-10-06T08:21:06.204929Z"
    },
    "papermill": {
     "duration": 13.967601,
     "end_time": "2024-10-06T08:21:06.208545",
     "exception": false,
     "start_time": "2024-10-06T08:20:52.240944",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "import tensorflow as tf\n",
    "from tensorflow import keras\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "pad_sequences = keras.preprocessing.sequence.pad_sequences"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "48e89d47",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T08:21:06.220997Z",
     "iopub.status.busy": "2024-10-06T08:21:06.220367Z",
     "iopub.status.idle": "2024-10-06T08:21:12.483836Z",
     "shell.execute_reply": "2024-10-06T08:21:12.483028Z"
    },
    "papermill": {
     "duration": 6.272464,
     "end_time": "2024-10-06T08:21:12.486410",
     "exception": false,
     "start_time": "2024-10-06T08:21:06.213946",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/imdb.npz\n",
      "\u001b[1m17464789/17464789\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 0us/step\n"
     ]
    }
   ],
   "source": [
    "imdb = keras.datasets.imdb\n",
    "(train_data, train_labels), (test_data, test_labels) = imdb.load_data(num_words=10000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "bf2cb766",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T08:21:12.500567Z",
     "iopub.status.busy": "2024-10-06T08:21:12.500272Z",
     "iopub.status.idle": "2024-10-06T08:21:12.510373Z",
     "shell.execute_reply": "2024-10-06T08:21:12.509557Z"
    },
    "papermill": {
     "duration": 0.019418,
     "end_time": "2024-10-06T08:21:12.512479",
     "exception": false,
     "start_time": "2024-10-06T08:21:12.493061",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "train_data, val_data, train_labels, val_labels = train_test_split(train_data, \n",
    "                                                                  train_labels, \n",
    "                                                                  test_size=0.30, \n",
    "                                                                  shuffle=True,\n",
    "                                                                  random_state=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "53cf0ee1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T08:21:12.525909Z",
     "iopub.status.busy": "2024-10-06T08:21:12.525611Z",
     "iopub.status.idle": "2024-10-06T08:21:13.413468Z",
     "shell.execute_reply": "2024-10-06T08:21:13.412670Z"
    },
    "id": "QVBYZDyfPdXl",
    "papermill": {
     "duration": 0.897301,
     "end_time": "2024-10-06T08:21:13.416044",
     "exception": false,
     "start_time": "2024-10-06T08:21:12.518743",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Downloading data from https://storage.googleapis.com/tensorflow/tf-keras-datasets/imdb_word_index.json\n",
      "\u001b[1m1641221/1641221\u001b[0m \u001b[32m━━━━━━━━━━━━━━━━━━━━\u001b[0m\u001b[37m\u001b[0m \u001b[1m1s\u001b[0m 0us/step\n"
     ]
    }
   ],
   "source": [
    "# A dictionary mapping words to an integer index\n",
    "word_index = imdb.get_word_index()\n",
    "\n",
    "# The first indices are reserved\n",
    "word_index = {k:(v+3) for k,v in word_index.items()} \n",
    "word_index[\"<PAD>\"] = 0\n",
    "word_index[\"<START>\"] = 1\n",
    "word_index[\"<UNK>\"] = 2  # unknown\n",
    "word_index[\"<UNUSED>\"] = 3\n",
    "\n",
    "reverse_word_index = dict([(value, key) for (key, value) in word_index.items()])\n",
    "\n",
    "def decode_review(text):\n",
    "    return ' '.join([reverse_word_index.get(i, '?') for i in text])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "fb297539",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T08:21:13.433298Z",
     "iopub.status.busy": "2024-10-06T08:21:13.432708Z",
     "iopub.status.idle": "2024-10-06T08:21:13.443895Z",
     "shell.execute_reply": "2024-10-06T08:21:13.443227Z"
    },
    "papermill": {
     "duration": 0.021141,
     "end_time": "2024-10-06T08:21:13.445796",
     "exception": false,
     "start_time": "2024-10-06T08:21:13.424655",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.models import Sequential\n",
    "from tensorflow.keras.layers import LeakyReLU\n",
    "from tensorflow.keras.layers import Activation\n",
    "from tensorflow.keras.optimizers import SGD, Adam\n",
    "\n",
    "from tensorflow.keras.callbacks import EarlyStopping, ModelCheckpoint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "0f73cdb7",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T08:21:13.461453Z",
     "iopub.status.busy": "2024-10-06T08:21:13.461157Z",
     "iopub.status.idle": "2024-10-06T08:21:14.389675Z",
     "shell.execute_reply": "2024-10-06T08:21:14.388706Z"
    },
    "id": "C5vN2mLLPd28",
    "papermill": {
     "duration": 0.938846,
     "end_time": "2024-10-06T08:21:14.392035",
     "exception": false,
     "start_time": "2024-10-06T08:21:13.453189",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "pad_length = 256\n",
    "\n",
    "train_data = pad_sequences(train_data,\n",
    "                           value=word_index[\"<PAD>\"],\n",
    "                           padding='post',\n",
    "                           maxlen=pad_length)\n",
    "\n",
    "val_data = pad_sequences(val_data,\n",
    "                         value=word_index[\"<PAD>\"],\n",
    "                         padding='post',\n",
    "                         maxlen=pad_length)\n",
    "\n",
    "test_data = pad_sequences(test_data,\n",
    "                          value=word_index[\"<PAD>\"],\n",
    "                          padding='post',\n",
    "                          maxlen=pad_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e1a67b74",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T08:21:14.407827Z",
     "iopub.status.busy": "2024-10-06T08:21:14.407511Z",
     "iopub.status.idle": "2024-10-06T08:21:14.414606Z",
     "shell.execute_reply": "2024-10-06T08:21:14.413826Z"
    },
    "papermill": {
     "duration": 0.017108,
     "end_time": "2024-10-06T08:21:14.416535",
     "exception": false,
     "start_time": "2024-10-06T08:21:14.399427",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "from tensorflow.keras.layers import Dense, Dropout\n",
    "from tensorflow.keras.layers import Flatten, RepeatVector, dot, multiply, Permute, Lambda\n",
    "\n",
    "def attention(layer):\n",
    "    # --- Attention is all you need --- #\n",
    "    units = layer.shape[-1]  # Use the last dimension\n",
    "    attention = Dense(1, activation='tanh')(layer)\n",
    "    attention = Flatten()(attention)\n",
    "    attention = tf.keras.activations.softmax(attention)\n",
    "    attention = RepeatVector(units)(attention)\n",
    "    attention = Permute([2, 1])(attention)\n",
    "    representation = multiply([layer, attention])\n",
    "    representation = Lambda(lambda x: tf.reduce_sum(x, axis=-2),\n",
    "                            output_shape=(units,))(representation)\n",
    "    # ---------------------------------- #\n",
    "    return representation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "d8ec4ad1",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T08:21:14.432360Z",
     "iopub.status.busy": "2024-10-06T08:21:14.431565Z",
     "iopub.status.idle": "2024-10-06T08:21:14.437111Z",
     "shell.execute_reply": "2024-10-06T08:21:14.436333Z"
    },
    "papermill": {
     "duration": 0.015449,
     "end_time": "2024-10-06T08:21:14.439016",
     "exception": false,
     "start_time": "2024-10-06T08:21:14.423567",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "def get_optimizer(option=0, learning_rate=0.001):\n",
    "    if option == 0:\n",
    "        return tf.keras.optimizers.Adam(learning_rate)\n",
    "    elif option == 1:\n",
    "        return tf.keras.optimizers.SGD(learning_rate, momentum=0.9, nesterov=True)\n",
    "    elif option == 2:\n",
    "        return tf.keras.optimizers.RMSprop(learning_rate) \n",
    "    else:\n",
    "        return tf.keras.optimizers.Adam(learning_rate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ac2ffd95",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T08:21:14.454280Z",
     "iopub.status.busy": "2024-10-06T08:21:14.453954Z",
     "iopub.status.idle": "2024-10-06T08:21:14.471672Z",
     "shell.execute_reply": "2024-10-06T08:21:14.470841Z"
    },
    "id": "1qiGmUv0dQCI",
    "papermill": {
     "duration": 0.027538,
     "end_time": "2024-10-06T08:21:14.473548",
     "exception": false,
     "start_time": "2024-10-06T08:21:14.446010",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "layers = keras.layers\n",
    "models = keras.models\n",
    "    \n",
    "def create_tunable_model(hp, vocab_size=10000, pad_length=256):\n",
    "\n",
    "    # Instantiate model params\n",
    "    embedding_size = hp.Int('embedding_size', min_value=8, max_value=512, step=8)\n",
    "    spatial_dropout = hp.Float('spatial_dropout', min_value=0, max_value=0.5, step=0.05)\n",
    "\n",
    "    conv_layers = hp.Int('conv_layers', min_value=1, max_value=5, step=1)\n",
    "    rnn_layers = hp.Int('rnn_layers', min_value=1, max_value=5, step=1)\n",
    "    dense_layers = hp.Int('dense_layers', min_value=1, max_value=3, step=1)\n",
    "\n",
    "    conv_filters = hp.Int('conv_filters', min_value=32, max_value=512, step=32)\n",
    "    conv_kernel = hp.Int('conv_kernel', min_value=1, max_value=8, step=1)\n",
    "\n",
    "    concat_dropout = hp.Float('concat_dropout', min_value=0, max_value=0.5, step=0.05)\n",
    "    dense_dropout = hp.Float('dense_dropout', min_value=0, max_value=0.5, step=0.05)\n",
    "\n",
    "    inputs = layers.Input(name='inputs',shape=[pad_length])\n",
    "    layer  = layers.Embedding(vocab_size, embedding_size)(inputs)\n",
    "    layer  = layers.SpatialDropout1D(spatial_dropout)(layer)\n",
    "\n",
    "    for l in range(conv_layers):\n",
    "        if l==0:\n",
    "            conv = layers.Conv1D(filters=conv_filters, kernel_size=conv_kernel, \n",
    "                                 padding='valid', kernel_initializer='he_uniform')(layer)\n",
    "        else:\n",
    "            conv = layers.Conv1D(filters=conv_filters, kernel_size=conv_kernel, \n",
    "                                 padding='valid', kernel_initializer='he_uniform')(conv) \n",
    "\n",
    "    avg_pool_conv = layers.GlobalAveragePooling1D()(conv)\n",
    "    max_pool_conv = layers.GlobalMaxPooling1D()(conv)\n",
    "\n",
    "    representations = list()\n",
    "    for l in range(rnn_layers):\n",
    "        \n",
    "        use_bidirectional = hp.Choice(f'use_bidirectional_{l}', values=[0, 1])\n",
    "        use_lstm = hp.Choice(f'use_lstm_{l}', values=[0, 1])\n",
    "        units = hp.Int(f'units_{l}', min_value=8, max_value=512, step=8)\n",
    "\n",
    "        if use_lstm == 1:\n",
    "            rnl = layers.LSTM\n",
    "        else:\n",
    "            rnl = layers.GRU\n",
    "\n",
    "        if use_bidirectional==1:\n",
    "            layer = layers.Bidirectional(rnl(units, return_sequences=True))(layer)\n",
    "        else:\n",
    "            layer = rnl(units, return_sequences=True)(layer)\n",
    "\n",
    "        representations.append(attention(layer))\n",
    "\n",
    "    layer = layers.concatenate(representations + [avg_pool_conv, max_pool_conv])\n",
    "    layer = layers.Dropout(concat_dropout)(layer)\n",
    "\n",
    "    for l in range(dense_layers):\n",
    "        dense_units = hp.Int(f'dense_units_{l}', min_value=8, max_value=512, step=8)\n",
    "        layer = layers.Dense(dense_units)(layer)\n",
    "        layer  = layers.LeakyReLU()(layer)\n",
    "        layer = layers.Dropout(dense_dropout)(layer)\n",
    "\n",
    "    layer  = layers.Dense(1, name='out_layer')(layer)\n",
    "    outputs  = layers.Activation('sigmoid')(layer)\n",
    "\n",
    "    model  = models.Model(inputs=inputs, outputs=outputs)\n",
    "\n",
    "    hp_learning_rate = hp.Choice('learning_rate', values=[0.002, 0.001, 0.0005])\n",
    "    optimizer_type = hp.Choice('optimizer', values=list(range(3)))\n",
    "    optimizer = get_optimizer(option=optimizer_type, learning_rate=hp_learning_rate)\n",
    "    \n",
    "    model.compile(optimizer=optimizer,\n",
    "                  loss='binary_crossentropy',\n",
    "                  metrics=['acc'])\n",
    "    \n",
    "    return model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "d755ca91",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T08:21:14.488703Z",
     "iopub.status.busy": "2024-10-06T08:21:14.488417Z",
     "iopub.status.idle": "2024-10-06T08:21:14.697684Z",
     "shell.execute_reply": "2024-10-06T08:21:14.696635Z"
    },
    "id": "v4AaBohkWErD",
    "papermill": {
     "duration": 0.219567,
     "end_time": "2024-10-06T08:21:14.700141",
     "exception": false,
     "start_time": "2024-10-06T08:21:14.480574",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "import keras_tuner as kt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "id": "81c80523",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T08:21:14.716881Z",
     "iopub.status.busy": "2024-10-06T08:21:14.716284Z",
     "iopub.status.idle": "2024-10-06T14:52:56.625984Z",
     "shell.execute_reply": "2024-10-06T14:52:56.624961Z"
    },
    "id": "3vznWJPcdQGt",
    "outputId": "57a3e20b-e211-4143-c553-d720c28040ac",
    "papermill": {
     "duration": 23501.920255,
     "end_time": "2024-10-06T14:52:56.628027",
     "exception": false,
     "start_time": "2024-10-06T08:21:14.707772",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Trial 100 Complete [00h 01m 24s]\n",
      "val_acc: 0.9017333388328552\n",
      "\n",
      "Best val_acc So Far: 0.9037333130836487\n",
      "Total elapsed time: 06h 31m 41s\n"
     ]
    }
   ],
   "source": [
    "tuner = kt.BayesianOptimization(hypermodel=create_tunable_model,\n",
    "                                objective='val_acc',\n",
    "                                max_trials=100,\n",
    "                                num_initial_points=3,\n",
    "                                directory='storage',\n",
    "                                project_name='imdb',\n",
    "                                seed=42)\n",
    "\n",
    "tuner.search(train_data, train_labels, \n",
    "             epochs=30,\n",
    "             batch_size=64, \n",
    "             validation_data=(val_data, val_labels),\n",
    "             shuffle=True,\n",
    "             verbose=2,\n",
    "             callbacks = [EarlyStopping('val_acc', patience=3, restore_best_weights=True)]\n",
    "             )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "3e36d641",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T14:52:56.643933Z",
     "iopub.status.busy": "2024-10-06T14:52:56.643639Z",
     "iopub.status.idle": "2024-10-06T14:52:56.647636Z",
     "shell.execute_reply": "2024-10-06T14:52:56.646841Z"
    },
    "papermill": {
     "duration": 0.014193,
     "end_time": "2024-10-06T14:52:56.649503",
     "exception": false,
     "start_time": "2024-10-06T14:52:56.635310",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "# use to visualize: https://github.com/lutzroeder/netron"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "985a5b9f",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T14:52:56.665375Z",
     "iopub.status.busy": "2024-10-06T14:52:56.664721Z",
     "iopub.status.idle": "2024-10-06T14:52:57.183129Z",
     "shell.execute_reply": "2024-10-06T14:52:57.182151Z"
    },
    "id": "vYf3mlBbVX25",
    "papermill": {
     "duration": 0.528764,
     "end_time": "2024-10-06T14:52:57.185482",
     "exception": false,
     "start_time": "2024-10-06T14:52:56.656718",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "best_hps = tuner.get_best_hyperparameters()[0]\n",
    "model = tuner.hypermodel.build(best_hps)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a534ae6c",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T14:52:57.201410Z",
     "iopub.status.busy": "2024-10-06T14:52:57.201052Z",
     "iopub.status.idle": "2024-10-06T14:52:57.205710Z",
     "shell.execute_reply": "2024-10-06T14:52:57.204795Z"
    },
    "papermill": {
     "duration": 0.014548,
     "end_time": "2024-10-06T14:52:57.207531",
     "exception": false,
     "start_time": "2024-10-06T14:52:57.192983",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'embedding_size': 512, 'spatial_dropout': 0.0, 'conv_layers': 1, 'rnn_layers': 5, 'dense_layers': 1, 'conv_filters': 512, 'conv_kernel': 8, 'concat_dropout': 0.0, 'dense_dropout': 0.0, 'use_bidirectional_0': 0, 'use_lstm_0': 0, 'units_0': 8, 'dense_units_0': 512, 'learning_rate': 0.0005, 'optimizer': 0, 'use_bidirectional_1': 1, 'use_lstm_1': 0, 'units_1': 8, 'dense_units_1': 8, 'dense_units_2': 512, 'use_bidirectional_2': 0, 'use_lstm_2': 0, 'units_2': 8, 'use_bidirectional_3': 0, 'use_lstm_3': 1, 'units_3': 512, 'use_bidirectional_4': 1, 'use_lstm_4': 0, 'units_4': 512}\n"
     ]
    }
   ],
   "source": [
    "print(best_hps.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "60b335b6",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T14:52:57.223696Z",
     "iopub.status.busy": "2024-10-06T14:52:57.223406Z",
     "iopub.status.idle": "2024-10-06T14:52:57.300029Z",
     "shell.execute_reply": "2024-10-06T14:52:57.299234Z"
    },
    "id": "upzApqVhdP-j",
    "papermill": {
     "duration": 0.087109,
     "end_time": "2024-10-06T14:52:57.302249",
     "exception": false,
     "start_time": "2024-10-06T14:52:57.215140",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\">Model: \"functional_3\"</span>\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1mModel: \"functional_3\"\u001b[0m\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\">┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓\n",
       "┃<span style=\"font-weight: bold\"> Layer (type)        </span>┃<span style=\"font-weight: bold\"> Output Shape      </span>┃<span style=\"font-weight: bold\">    Param # </span>┃<span style=\"font-weight: bold\"> Connected to      </span>┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩\n",
       "│ inputs (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">InputLayer</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ -                 │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ embedding_1         │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">512</span>)  │  <span style=\"color: #00af00; text-decoration-color: #00af00\">5,120,000</span> │ inputs[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]      │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Embedding</span>)         │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ spatial_dropout1d_1 │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">512</span>)  │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ embedding_1[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>] │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">SpatialDropout1D</span>)  │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ gru_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GRU</span>)         │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">8</span>)    │     <span style=\"color: #00af00; text-decoration-color: #00af00\">12,528</span> │ spatial_dropout1… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ bidirectional       │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)   │        <span style=\"color: #00af00; text-decoration-color: #00af00\">864</span> │ gru_1[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]       │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>)     │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ gru_3 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GRU</span>)         │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">8</span>)    │        <span style=\"color: #00af00; text-decoration-color: #00af00\">624</span> │ bidirectional[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ lstm (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">LSTM</span>)         │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">512</span>)  │  <span style=\"color: #00af00; text-decoration-color: #00af00\">1,067,008</span> │ gru_3[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]       │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ bidirectional_1     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1024</span>) │  <span style=\"color: #00af00; text-decoration-color: #00af00\">3,151,872</span> │ lstm[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]        │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Bidirectional</span>)     │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)    │          <span style=\"color: #00af00; text-decoration-color: #00af00\">9</span> │ gru_1[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]       │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_3 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)    │         <span style=\"color: #00af00; text-decoration-color: #00af00\">17</span> │ bidirectional[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_4 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)    │          <span style=\"color: #00af00; text-decoration-color: #00af00\">9</span> │ gru_3[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]       │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_5 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)    │        <span style=\"color: #00af00; text-decoration-color: #00af00\">513</span> │ lstm[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]        │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_6 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)    │      <span style=\"color: #00af00; text-decoration-color: #00af00\">1,025</span> │ bidirectional_1[<span style=\"color: #00af00; text-decoration-color: #00af00\">…</span> │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ flatten_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Flatten</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ dense_2[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]     │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ flatten_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Flatten</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ dense_3[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]     │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ flatten_3 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Flatten</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ dense_4[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]     │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ flatten_4 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Flatten</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ dense_5[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]     │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ flatten_5 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Flatten</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ dense_6[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]     │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ softmax_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Softmax</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ flatten_1[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ softmax_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Softmax</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ flatten_2[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ softmax_3 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Softmax</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ flatten_3[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ softmax_4 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Softmax</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ flatten_4[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ softmax_5 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Softmax</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ flatten_5[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ repeat_vector_1     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">8</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)    │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ softmax_1[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">RepeatVector</span>)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ repeat_vector_2     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)   │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ softmax_2[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">RepeatVector</span>)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ repeat_vector_3     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">8</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)    │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ softmax_3[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">RepeatVector</span>)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ repeat_vector_4     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">512</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>)  │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ softmax_4[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">RepeatVector</span>)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ repeat_vector_5     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1024</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>) │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ softmax_5[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">RepeatVector</span>)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ permute_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Permute</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">8</span>)    │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ repeat_vector_1[<span style=\"color: #00af00; text-decoration-color: #00af00\">…</span> │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ permute_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Permute</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)   │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ repeat_vector_2[<span style=\"color: #00af00; text-decoration-color: #00af00\">…</span> │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ permute_3 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Permute</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">8</span>)    │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ repeat_vector_3[<span style=\"color: #00af00; text-decoration-color: #00af00\">…</span> │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ permute_4 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Permute</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">512</span>)  │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ repeat_vector_4[<span style=\"color: #00af00; text-decoration-color: #00af00\">…</span> │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ permute_5 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Permute</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1024</span>) │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ repeat_vector_5[<span style=\"color: #00af00; text-decoration-color: #00af00\">…</span> │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ multiply_1          │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">8</span>)    │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ gru_1[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>],      │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Multiply</span>)          │                   │            │ permute_1[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ multiply_2          │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)   │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ bidirectional[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]… │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Multiply</span>)          │                   │            │ permute_2[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ multiply_3          │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">8</span>)    │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ gru_3[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>],      │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Multiply</span>)          │                   │            │ permute_3[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ multiply_4          │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">512</span>)  │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ lstm[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>],       │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Multiply</span>)          │                   │            │ permute_4[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ multiply_5          │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">256</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1024</span>) │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ bidirectional_1[<span style=\"color: #00af00; text-decoration-color: #00af00\">…</span> │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Multiply</span>)          │                   │            │ permute_5[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Conv1D</span>)   │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">249</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">512</span>)  │  <span style=\"color: #00af00; text-decoration-color: #00af00\">2,097,664</span> │ spatial_dropout1… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ lambda_1 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Lambda</span>)   │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">8</span>)         │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ multiply_1[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]  │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ lambda_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Lambda</span>)   │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">16</span>)        │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ multiply_2[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]  │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ lambda_3 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Lambda</span>)   │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">8</span>)         │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ multiply_3[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]  │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ lambda_4 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Lambda</span>)   │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">512</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ multiply_4[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]  │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ lambda_5 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Lambda</span>)   │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1024</span>)      │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ multiply_5[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]  │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ global_average_poo… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">512</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ conv1d_1[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]    │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GlobalAveragePool…</span> │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ global_max_pooling… │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">512</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ conv1d_1[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]    │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">GlobalMaxPooling1…</span> │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ concatenate_1       │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">2592</span>)      │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ lambda_1[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>],   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Concatenate</span>)       │                   │            │ lambda_2[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>],   │\n",
       "│                     │                   │            │ lambda_3[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>],   │\n",
       "│                     │                   │            │ lambda_4[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>],   │\n",
       "│                     │                   │            │ lambda_5[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>],   │\n",
       "│                     │                   │            │ global_average_p… │\n",
       "│                     │                   │            │ global_max_pooli… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dropout_2 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">2592</span>)      │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ concatenate_1[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_7 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)     │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">512</span>)       │  <span style=\"color: #00af00; text-decoration-color: #00af00\">1,327,616</span> │ dropout_2[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ leaky_re_lu_1       │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">512</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ dense_7[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]     │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">LeakyReLU</span>)         │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dropout_3 (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dropout</span>) │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">512</span>)       │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ leaky_re_lu_1[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ out_layer (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Dense</span>)   │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)         │        <span style=\"color: #00af00; text-decoration-color: #00af00\">513</span> │ dropout_3[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ activation_1        │ (<span style=\"color: #00d7ff; text-decoration-color: #00d7ff\">None</span>, <span style=\"color: #00af00; text-decoration-color: #00af00\">1</span>)         │          <span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> │ out_layer[<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>][<span style=\"color: #00af00; text-decoration-color: #00af00\">0</span>]   │\n",
       "│ (<span style=\"color: #0087ff; text-decoration-color: #0087ff\">Activation</span>)        │                   │            │                   │\n",
       "└─────────────────────┴───────────────────┴────────────┴───────────────────┘\n",
       "</pre>\n"
      ],
      "text/plain": [
       "┏━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━┓\n",
       "┃\u001b[1m \u001b[0m\u001b[1mLayer (type)       \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mOutput Shape     \u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1m   Param #\u001b[0m\u001b[1m \u001b[0m┃\u001b[1m \u001b[0m\u001b[1mConnected to     \u001b[0m\u001b[1m \u001b[0m┃\n",
       "┡━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━┩\n",
       "│ inputs (\u001b[38;5;33mInputLayer\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ -                 │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ embedding_1         │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m512\u001b[0m)  │  \u001b[38;5;34m5,120,000\u001b[0m │ inputs[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]      │\n",
       "│ (\u001b[38;5;33mEmbedding\u001b[0m)         │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ spatial_dropout1d_1 │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m512\u001b[0m)  │          \u001b[38;5;34m0\u001b[0m │ embedding_1[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m] │\n",
       "│ (\u001b[38;5;33mSpatialDropout1D\u001b[0m)  │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ gru_1 (\u001b[38;5;33mGRU\u001b[0m)         │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m8\u001b[0m)    │     \u001b[38;5;34m12,528\u001b[0m │ spatial_dropout1… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ bidirectional       │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m16\u001b[0m)   │        \u001b[38;5;34m864\u001b[0m │ gru_1[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]       │\n",
       "│ (\u001b[38;5;33mBidirectional\u001b[0m)     │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ gru_3 (\u001b[38;5;33mGRU\u001b[0m)         │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m8\u001b[0m)    │        \u001b[38;5;34m624\u001b[0m │ bidirectional[\u001b[38;5;34m0\u001b[0m]… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ lstm (\u001b[38;5;33mLSTM\u001b[0m)         │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m512\u001b[0m)  │  \u001b[38;5;34m1,067,008\u001b[0m │ gru_3[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]       │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ bidirectional_1     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m1024\u001b[0m) │  \u001b[38;5;34m3,151,872\u001b[0m │ lstm[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]        │\n",
       "│ (\u001b[38;5;33mBidirectional\u001b[0m)     │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_2 (\u001b[38;5;33mDense\u001b[0m)     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m1\u001b[0m)    │          \u001b[38;5;34m9\u001b[0m │ gru_1[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]       │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_3 (\u001b[38;5;33mDense\u001b[0m)     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m1\u001b[0m)    │         \u001b[38;5;34m17\u001b[0m │ bidirectional[\u001b[38;5;34m0\u001b[0m]… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_4 (\u001b[38;5;33mDense\u001b[0m)     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m1\u001b[0m)    │          \u001b[38;5;34m9\u001b[0m │ gru_3[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]       │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_5 (\u001b[38;5;33mDense\u001b[0m)     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m1\u001b[0m)    │        \u001b[38;5;34m513\u001b[0m │ lstm[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]        │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_6 (\u001b[38;5;33mDense\u001b[0m)     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m1\u001b[0m)    │      \u001b[38;5;34m1,025\u001b[0m │ bidirectional_1[\u001b[38;5;34m…\u001b[0m │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ flatten_1 (\u001b[38;5;33mFlatten\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ dense_2[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]     │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ flatten_2 (\u001b[38;5;33mFlatten\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ dense_3[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]     │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ flatten_3 (\u001b[38;5;33mFlatten\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ dense_4[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]     │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ flatten_4 (\u001b[38;5;33mFlatten\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ dense_5[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]     │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ flatten_5 (\u001b[38;5;33mFlatten\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ dense_6[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]     │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ softmax_1 (\u001b[38;5;33mSoftmax\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ flatten_1[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ softmax_2 (\u001b[38;5;33mSoftmax\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ flatten_2[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ softmax_3 (\u001b[38;5;33mSoftmax\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ flatten_3[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ softmax_4 (\u001b[38;5;33mSoftmax\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ flatten_4[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ softmax_5 (\u001b[38;5;33mSoftmax\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ flatten_5[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ repeat_vector_1     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m8\u001b[0m, \u001b[38;5;34m256\u001b[0m)    │          \u001b[38;5;34m0\u001b[0m │ softmax_1[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mRepeatVector\u001b[0m)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ repeat_vector_2     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m16\u001b[0m, \u001b[38;5;34m256\u001b[0m)   │          \u001b[38;5;34m0\u001b[0m │ softmax_2[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mRepeatVector\u001b[0m)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ repeat_vector_3     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m8\u001b[0m, \u001b[38;5;34m256\u001b[0m)    │          \u001b[38;5;34m0\u001b[0m │ softmax_3[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mRepeatVector\u001b[0m)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ repeat_vector_4     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m512\u001b[0m, \u001b[38;5;34m256\u001b[0m)  │          \u001b[38;5;34m0\u001b[0m │ softmax_4[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mRepeatVector\u001b[0m)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ repeat_vector_5     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1024\u001b[0m, \u001b[38;5;34m256\u001b[0m) │          \u001b[38;5;34m0\u001b[0m │ softmax_5[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mRepeatVector\u001b[0m)      │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ permute_1 (\u001b[38;5;33mPermute\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m8\u001b[0m)    │          \u001b[38;5;34m0\u001b[0m │ repeat_vector_1[\u001b[38;5;34m…\u001b[0m │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ permute_2 (\u001b[38;5;33mPermute\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m16\u001b[0m)   │          \u001b[38;5;34m0\u001b[0m │ repeat_vector_2[\u001b[38;5;34m…\u001b[0m │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ permute_3 (\u001b[38;5;33mPermute\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m8\u001b[0m)    │          \u001b[38;5;34m0\u001b[0m │ repeat_vector_3[\u001b[38;5;34m…\u001b[0m │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ permute_4 (\u001b[38;5;33mPermute\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m512\u001b[0m)  │          \u001b[38;5;34m0\u001b[0m │ repeat_vector_4[\u001b[38;5;34m…\u001b[0m │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ permute_5 (\u001b[38;5;33mPermute\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m1024\u001b[0m) │          \u001b[38;5;34m0\u001b[0m │ repeat_vector_5[\u001b[38;5;34m…\u001b[0m │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ multiply_1          │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m8\u001b[0m)    │          \u001b[38;5;34m0\u001b[0m │ gru_1[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m],      │\n",
       "│ (\u001b[38;5;33mMultiply\u001b[0m)          │                   │            │ permute_1[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ multiply_2          │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m16\u001b[0m)   │          \u001b[38;5;34m0\u001b[0m │ bidirectional[\u001b[38;5;34m0\u001b[0m]… │\n",
       "│ (\u001b[38;5;33mMultiply\u001b[0m)          │                   │            │ permute_2[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ multiply_3          │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m8\u001b[0m)    │          \u001b[38;5;34m0\u001b[0m │ gru_3[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m],      │\n",
       "│ (\u001b[38;5;33mMultiply\u001b[0m)          │                   │            │ permute_3[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ multiply_4          │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m512\u001b[0m)  │          \u001b[38;5;34m0\u001b[0m │ lstm[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m],       │\n",
       "│ (\u001b[38;5;33mMultiply\u001b[0m)          │                   │            │ permute_4[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ multiply_5          │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m256\u001b[0m, \u001b[38;5;34m1024\u001b[0m) │          \u001b[38;5;34m0\u001b[0m │ bidirectional_1[\u001b[38;5;34m…\u001b[0m │\n",
       "│ (\u001b[38;5;33mMultiply\u001b[0m)          │                   │            │ permute_5[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ conv1d_1 (\u001b[38;5;33mConv1D\u001b[0m)   │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m249\u001b[0m, \u001b[38;5;34m512\u001b[0m)  │  \u001b[38;5;34m2,097,664\u001b[0m │ spatial_dropout1… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ lambda_1 (\u001b[38;5;33mLambda\u001b[0m)   │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m8\u001b[0m)         │          \u001b[38;5;34m0\u001b[0m │ multiply_1[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]  │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ lambda_2 (\u001b[38;5;33mLambda\u001b[0m)   │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m16\u001b[0m)        │          \u001b[38;5;34m0\u001b[0m │ multiply_2[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]  │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ lambda_3 (\u001b[38;5;33mLambda\u001b[0m)   │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m8\u001b[0m)         │          \u001b[38;5;34m0\u001b[0m │ multiply_3[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]  │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ lambda_4 (\u001b[38;5;33mLambda\u001b[0m)   │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m512\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ multiply_4[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]  │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ lambda_5 (\u001b[38;5;33mLambda\u001b[0m)   │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1024\u001b[0m)      │          \u001b[38;5;34m0\u001b[0m │ multiply_5[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]  │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ global_average_poo… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m512\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ conv1d_1[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]    │\n",
       "│ (\u001b[38;5;33mGlobalAveragePool…\u001b[0m │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ global_max_pooling… │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m512\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ conv1d_1[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]    │\n",
       "│ (\u001b[38;5;33mGlobalMaxPooling1…\u001b[0m │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ concatenate_1       │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m2592\u001b[0m)      │          \u001b[38;5;34m0\u001b[0m │ lambda_1[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m],   │\n",
       "│ (\u001b[38;5;33mConcatenate\u001b[0m)       │                   │            │ lambda_2[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m],   │\n",
       "│                     │                   │            │ lambda_3[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m],   │\n",
       "│                     │                   │            │ lambda_4[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m],   │\n",
       "│                     │                   │            │ lambda_5[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m],   │\n",
       "│                     │                   │            │ global_average_p… │\n",
       "│                     │                   │            │ global_max_pooli… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dropout_2 (\u001b[38;5;33mDropout\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m2592\u001b[0m)      │          \u001b[38;5;34m0\u001b[0m │ concatenate_1[\u001b[38;5;34m0\u001b[0m]… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dense_7 (\u001b[38;5;33mDense\u001b[0m)     │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m512\u001b[0m)       │  \u001b[38;5;34m1,327,616\u001b[0m │ dropout_2[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ leaky_re_lu_1       │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m512\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ dense_7[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]     │\n",
       "│ (\u001b[38;5;33mLeakyReLU\u001b[0m)         │                   │            │                   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ dropout_3 (\u001b[38;5;33mDropout\u001b[0m) │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m512\u001b[0m)       │          \u001b[38;5;34m0\u001b[0m │ leaky_re_lu_1[\u001b[38;5;34m0\u001b[0m]… │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ out_layer (\u001b[38;5;33mDense\u001b[0m)   │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)         │        \u001b[38;5;34m513\u001b[0m │ dropout_3[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "├─────────────────────┼───────────────────┼────────────┼───────────────────┤\n",
       "│ activation_1        │ (\u001b[38;5;45mNone\u001b[0m, \u001b[38;5;34m1\u001b[0m)         │          \u001b[38;5;34m0\u001b[0m │ out_layer[\u001b[38;5;34m0\u001b[0m][\u001b[38;5;34m0\u001b[0m]   │\n",
       "│ (\u001b[38;5;33mActivation\u001b[0m)        │                   │            │                   │\n",
       "└─────────────────────┴───────────────────┴────────────┴───────────────────┘\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Total params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">12,780,262</span> (48.75 MB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Total params: \u001b[0m\u001b[38;5;34m12,780,262\u001b[0m (48.75 MB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">12,780,262</span> (48.75 MB)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Trainable params: \u001b[0m\u001b[38;5;34m12,780,262\u001b[0m (48.75 MB)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<pre style=\"white-space:pre;overflow-x:auto;line-height:normal;font-family:Menlo,'DejaVu Sans Mono',consolas,'Courier New',monospace\"><span style=\"font-weight: bold\"> Non-trainable params: </span><span style=\"color: #00af00; text-decoration-color: #00af00\">0</span> (0.00 B)\n",
       "</pre>\n"
      ],
      "text/plain": [
       "\u001b[1m Non-trainable params: \u001b[0m\u001b[38;5;34m0\u001b[0m (0.00 B)\n"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "model.summary()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "37093206",
   "metadata": {
    "execution": {
     "iopub.execute_input": "2024-10-06T14:52:57.321353Z",
     "iopub.status.busy": "2024-10-06T14:52:57.321045Z",
     "iopub.status.idle": "2024-10-06T14:52:57.447893Z",
     "shell.execute_reply": "2024-10-06T14:52:57.447029Z"
    },
    "papermill": {
     "duration": 0.138828,
     "end_time": "2024-10-06T14:52:57.450223",
     "exception": false,
     "start_time": "2024-10-06T14:52:57.311395",
     "status": "completed"
    },
    "tags": []
   },
   "outputs": [],
   "source": [
    "model.save(\"best_model.h5\")"
   ]
  }
 ],
 "metadata": {
  "kaggle": {
   "accelerator": "gpu",
   "dataSources": [],
   "isGpuEnabled": true,
   "isInternetEnabled": true,
   "language": "python",
   "sourceType": "notebook"
  },
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 23530.628583,
   "end_time": "2024-10-06T14:53:00.145951",
   "environment_variables": {},
   "exception": null,
   "input_path": "__notebook__.ipynb",
   "output_path": "__notebook__.ipynb",
   "parameters": {},
   "start_time": "2024-10-06T08:20:49.517368",
   "version": "2.6.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
